""" AnyBackup is a windows specific backup program to keep multiple volumes in sync with multiple volumes
    Copyright (C) 2011  Andrew Hopkinson

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

    Exported Classes:

    backupManager -- the backup class for AnyBackup, it contains the drive objects, meta data,
                     and all the backup/restore logic
    """

import ConfigParser
import shutil
import os
import stat
import hashlib
import logging
import itertools
from time import sleep
import Pyro4
from sql import SQLLite, AbortTransaction, getHomedir
from enum import AEnums
import DriveUtil
from mixins import messagingMixin, print_timing, backupMixin
import traceback
import datetime
import re
import subprocess


messagingQueue = None
errorQueue = None
errorAnswerQueue = None

logger = logging.getLogger('AnyBackup')


def file_cmp(a, b):
    return cmp(a[AEnums.PREFIX].lower(), b[AEnums.PREFIX].lower()) or \
        cmp(a[AEnums.PATH].lower(), b[AEnums.PATH].lower()) or cmp(a[AEnums.NAME].lower(), b[AEnums.NAME].lower())


class AbortCopy(Exception):
    pass


class AbortDrive(Exception):
    pass


class backupManager(backupMixin, messagingMixin):
    """ the backup class for AnyBackup, it contains the drive objects, meta data and all the backup/restore logic """
    BACKUP = AEnums.BACKUP
    CONTENT = AEnums.CONTENT
    ALL = 4
    DUPLICATE = 8
    UNNAMED = 7
    INVALIDCONFIG = 6
    MISSING = 9
    BALANCED = 'balanced'
    STICKY = 'sticky'
    version = '1.0.6'
    DEFAULTS = {'drive-selection': BALANCED,
                'pyro-port': 6500, 'nag-length': '0'}

    def __init__(self, setName=None, guest=None):
        """

        :param setName: The set name this backupManager should be working off of -- impacts what data it grabs from
                        sqlite and what dirs it scans and backups
        """
        self._setName = setName
        self._skipList = []
        self._validExt = []
        self._pendingWriteDirs = {}
        self._pendingWrite = {}
        self.r = SQLLite(guest=guest)
        self.hd = getHomedir()
        if not self._setName:
            self._setName = self.r.getSets()[0]
        self.readConfigs()
        self._backupDrives = []
        self._contentDrives = []
        self.kill = None
        self.resetFileVars()

    def addSet(self, setName):
        self.r.addSet(setName)

    def removeSet(self, setName):
        self.r.removeSet(setName)

    def getSetName(self):
        return self._setName

    def setName(self, setName):
        self._setName = setName

    def setErrorQueue(self, queue):
        global errorQueue
        errorQueue = queue

    def isBackupMode(self, drives):
        if drives:
            for driveRef in drives:
                if driveRef[AEnums.TYPE] != AEnums.BACKUP:
                    return False
            return True
        return False

    def setErrorAnswerQueue(self, queue):
        global errorAnswerQueue
        errorAnswerQueue = queue

    def sendError(self, error):
        if errorQueue:
            errorQueue.put(error, 1)
        else:
            query = raw_input('{0} not connected, connect and proceed? (y/n):'.format(error))
            if query == 'y':
                self.answer = True
            else:
                self.answer = False

    def getErrorAnswer(self):
        if not errorAnswerQueue:
            try:
                return self.answer
            except AttributeError:
                return False
        else:
            while errorAnswerQueue.empty():
                sleep(1)
            return errorAnswerQueue.get_nowait()

    def setMessagingQueue(self, queue):
        global messagingQueue
        messagingQueue = queue
        for drive in self._backupDrives + self._contentDrives:
            drive.setMessagingQueue(queue)

    def toggleDrive(self, driveRef):
        if driveRef.isBackup():
            self._backupDrives.remove(driveRef)
            self._contentDrives.append(driveRef)
            driveRef.setIsBackup(False)
        else:
            self._contentDrives.remove(driveRef)
            self._backupDrives.append(driveRef)
            driveRef.setIsBackup(True)
        return True

    def readConfigs(self):
        config = ConfigParser.SafeConfigParser(self.DEFAULTS)
        path = os.path.join(self.hd, 'AnyBackup.cfg')
        if os.path.exists(path):
            # noinspection PyBroadException
            try:
                config.read(path)
                backupMode = config.get('anybackup', 'drive-selection')
                if backupMode == self.BALANCED or backupMode == self.STICKY:
                    self.backupMode = backupMode
                else:
                    self.backupMode = self.BALANCED
                    return self.INVALIDCONFIG
                self.pyroPort = int(config.get('anybackup', 'pyro-port'))
                self.nagLength = int(config.get('anybackup', 'nag-length'))
            except:
                self.backupMode = self.BALANCED
                self.pyroPort = 6500
                self.nagLength = 0
                return self.INVALIDCONFIG
        else:
            config.add_section('anybackup')
            config.set('anybackup', 'drive-selection', self.BALANCED)
            config.set('anybackup', 'pyro-port', '6500')
            config.set('anybackup', 'nag-length', '0')
            self.backupMode = self.BALANCED
            self.pyroPort = 6500
            with open(path, 'w') as f:
                config.write(f)
        return True

    def getBackupMode(self):
        return self.backupMode

    def sortFiles(self, files):
        files.sort(file_cmp)
        return files

    def getRootFile(self, driveRef):
        return self.r.getRootFile(driveRef)

    def getFiles(self, fileRef, dirs=False):
        if isinstance(fileRef, dict) and AEnums.ID in fileRef:
            return self.r.getFiles(fileRef[AEnums.ID], dirs)
        elif type(fileRef) in [str, unicode]:
            return self.r.getFiles(fileRef, dirs)
        else:
            return self.r.getRootFile(fileRef, dirs)

    def getFileName(self, fileRef):
        return fileRef[AEnums.NAME]

    def getDir(self, fileRef):
        return fileRef[AEnums.PATH]

    def isDir(self, fileRef):
        return fileRef[AEnums.DIR]

    def getDrive(self, fileRef):
        return self.r.getDriveByFile(fileRef)

    def remoteCall(self, drive, kill=None):
        proxy = Pyro4.Proxy("PYRO:anybackup.indexserver@%s:%d" % (
                            drive[AEnums.REMOTEHOST], self.pyroPort))
        indexServer = Pyro4.async(proxy)
        commDict = self.buildCommDict(drive)
        result = indexServer.index(commDict)
        while not result.ready:
            if self.shouldExit(kill):
                proxy._pyroRelease()
                raise AbortTransaction('Aborting remote indexing...')
            result.wait(5)
            self.fileCount = proxy.count()
        logger.info('Remote result ready, releasing pyro.')
        proxy._pyroRelease()
        logger.info('Pyro released, returning result.')
        return result.value

    def buildCommDict(self, drive):
        return {AEnums.SEP: '\\', AEnums.NAME: drive[AEnums.NAME], AEnums.SERIAL: drive[AEnums.SERIAL],
                AEnums.PATHS: self.r.getPaths(drive), AEnums.REMOTEPATH: drive[AEnums.REMOTEPATH],
                AEnums.SKIPLIST: self.r.getSkipList(self._setName),
                AEnums.VALIDEXTS: self.r.getValidExt(self._setName), AEnums.SETNAME: drive[AEnums.SETNAME]}

    def remoteIndexDrive(self, drive, kill=None):
        if not AEnums.REMOTEHOST in drive or not AEnums.REMOTEPATH in drive:
            return False
        with self.r.batch():
            self.fileCount = 0
            self.r.removeFiles([drive])
            result = self.remoteCall(drive, kill=kill)
            if result == AEnums.NOTEXIST:
                return result
            for fileDict in result[AEnums.FILES]:
                self.r.addFile(drive, fileDict, fileDict[AEnums.ID] in result[AEnums.ROOT])
            self.r.updateFreeSpace(drive)
            return result
        # noinspection PyUnreachableCode
        # This is reached in the event of an abort transaction exception
        return AEnums.ABORT

    def refreshDrive(self, driveRef, kill=None):
        with self.r.batch():
            return self.indexDrives([driveRef], kill=kill)

    def refreshDrives(self, driveType):
        with self.r.batch():
            if driveType is None:
                driveType = backupManager.ALL
            drives = []
            if driveType == backupManager.ALL or driveType == backupManager.BACKUP:
                drives += self.getBackupDrives()
            if driveType == backupManager.ALL or driveType == backupManager.CONTENT:
                drives += self.getContentDrives()
            return self.indexDrives(drives)

    def getSkipList(self):
        return self.r.getSkipList(self._setName)

    def addSkip(self, skip):
        if skip:
            self.r.addSkip(skip, self._setName)
            logger.info('Add to skip list: {0}'.format(skip))

    def setSkipList(self, skip):
        if skip:
            self.r.setSkip(skip, self._setName)

    def getValidExt(self):
        return self.r.getValidExt(self._setName)

    def addValidExt(self, ext):
        if ext:
            self.r.addValidExt(ext, self._setName)

    def setValidExt(self, ext):
        if ext:
            self.r.setValidExt(ext, self._setName)

    def addBackupDrive(self, letter, index=True, kill=None):
        return self.addDrive(letter, AEnums.BACKUP, index, kill=kill)

    def addContentDrive(self, letter, paths, index=True, kill=None):
        return self.addDrive(letter, AEnums.CONTENT, index, paths, kill=kill)

    def removeDrives(self, drivesIn):
        if not drivesIn:
            logger.error('Undefined drive reference, check your selection')
            return False
        with self.r.batch():
            for d in drivesIn:
                self.r.removeDrive(d)

    def getBackupDrives(self):
        return self.r.getDrives(AEnums.BACKUP, self._setName)

    def getContentDrives(self):
        return self.r.getDrives(AEnums.CONTENT, self._setName)

    def toggleLock(self, drives):
        allLocked = True
        for driveRef in drives:
            if not self.r.isLocked(driveRef):
                allLocked = False
                break
        self.r.lockDrives(drives, not allLocked)

    def searchFiles(self, regex):
        return self.r.search(regex, self._setName)

    def uniqueCheck(self, name, serial):
        for d in self.getContentDrives() + self.getBackupDrives():
            if d[AEnums.NAME] == name and d[AEnums.SERIAL] == serial:
                logger.error("You've already added " + name + " - " + serial + "!")
                return False
            elif d[AEnums.NAME] == name:
                logger.warn(
                    "WARNING: " + name + " -- You've got two drives with the same Volume Name -- " +
                    "this is a bad idea! (Repent!)")
        return True

    def addDrive(self, letter, driveType, index=True, paths=None, kill=None):
        with self.r.batch():
            name = DriveUtil.getName(letter)
            serial = DriveUtil.getSerial(letter)
            logger.info('Adding drive {0}:'.format(letter))
            if not DriveUtil.exists(letter):
                logger.error('Drive {0}:\\ does not exist!'.format(letter))
                return AEnums.MISSING
            if not name:
                logger.error('Drive {0}:\\ must be named before adding!'.format(letter))
                return AEnums.UNNAMED
            if driveType == AEnums.BACKUP:
                p = self.createPath(letter + ":", self._setName)
                if not os.path.exists(p):
                    os.mkdir(p)
            if name and serial:
                if not self.uniqueCheck(name, serial):
                    logger.error('Drive {0}:\\ has already been added!'.format(letter))
                    return AEnums.DUPLICATE
                d = {
                    AEnums.NAME: name,
                    AEnums.SERIAL: serial,
                    AEnums.TYPE: driveType,
                    AEnums.SETNAME: self._setName,
                }
                self.r.addDrive(d, paths=paths)
                if index:
                    self.indexDrives([d], kill=kill)
                logger.info('Drive {0}:\\ added'.format(letter))
                self.r.updateFreeSpace(d)
                return True
        return False

    def isConnected(self, drive):
        if AEnums.NAME in drive and AEnums.SERIAL in drive:
            return DriveUtil.getLetter(drive[AEnums.NAME], drive[AEnums.SERIAL])

    def cleanUp(self, d):
        backupLetter = DriveUtil.getLetter(d[AEnums.NAME], d[AEnums.SERIAL])
        if not backupLetter:
            return
        dirName = self.createPath(backupLetter + ":\\AnyBackup")
        for filePath in os.listdir(unicode(dirName)):
            path = self.createPath(dirName, filePath)
            if os.path.isdir(self.toFullPath(path)):
                self.cleanDir(path)

    def cleanDir(self, path):
        """
        Recursively cleans up empty folders in a given path
        :param path: path to clean search for empty folders
        :return:
        """
        files = os.listdir(self.toFullPath(path))
        if not files:
            try:
                os.chmod(self.toFullPath(path), stat.S_IWRITE)
                os.rmdir(self.toFullPath(path))
            except Exception, e:
                logger.error('Could not remove empty path {0}: {1}'.format(path.encode('utf-16'), str(e)))
            return
        for fileName in files:
            filepath = self.createPath(path, fileName)
            if os.path.isdir(self.toFullPath(filepath)):
                self.cleanDir(filepath)
        if not os.listdir(self.toFullPath(path)):
            self.cleanDir(path)

    def indexDrives(self, drives, kill=None):
        for d in drives:
            if self.shouldExit(kill):
                raise AbortTransaction('Aborting index...')
            letter = DriveUtil.getLetter(d[AEnums.NAME], d[AEnums.SERIAL])
            if letter:
                self.r.removeFiles([d])
                self._indexDrive(d, letter, kill=kill)
            else:
                return AEnums.MISSING
        return True

    def _indexDrive(self, d, letter, kill=None):
        self.fileCount = 0
        validext = self.compileValidExt(self.r.getValidExt(d[AEnums.SETNAME]))
        skip = self.compileSkipList(self.r.getSkipList(d[AEnums.SETNAME]))
        if d[AEnums.TYPE] == AEnums.BACKUP:
            self.cleanUp(d)
            fileDict = self.index("", d, letter, validext, skip, root=True, kill=kill)
            if fileDict:
                self.r.addFile(d, fileDict, root=True)
        else:
            files = []
            for path in self.r.getPaths(d):
                files.append(self.index(path, d, letter, validext, skip, root=True, kill=kill))
            [self.r.addFile(d, x, root=True) for x in files]

    def sendMessage(self, message):
        if messagingQueue:
            messagingQueue.put(message, 1)
        else:
            print message

    def buildDir(self, drive, letter, path):
        if drive[AEnums.TYPE] == AEnums.BACKUP:
            dirName = self.createPath(letter + ":", self._setName, path)
        else:
            dirName = self.createPath(letter + ":", path)
        # logger.debug('Indexing: {0}'.format(dirName.encode('utf-8')))
        while dirName.endswith(os.sep):
            dirName = dirName.rstrip(os.sep)
        if not os.path.isdir(dirName):
            logger.warn("Can only call index on a directory!")
            return None
        if dirName.endswith(':'):
            dirName += os.sep
        return dirName

    def index(self, path, drive, letter, validext, skip, root=False, kill=None):
        """
        Indexes a given directory path for a given drive ref, letter with respect to the valid extension and skip
        regex lists -- optionally can be told to treat this as a root directory
        :param path: path to index
        :param drive: drive this path belongs to
        :param letter: drive's current letter
        :param validext: list of valid extensions (defaults to all)
        :param skip: list of regex's to skip for filepaths
        :param root: is this a root directory
        :return: a file reference for the directory
        """
        if self.shouldExit(kill):
            raise AbortTransaction('Aborting...')
        dirName = self.buildDir(drive, letter, path)
        if not dirName:
            return None
        files = self.getIndexFiles(dirName)
        fd = self.createDirRef(drive, path, dirName, root)
        passedFiles = []
        for foundFile in files[:]:
            if self.shouldExit(kill):
                raise AbortTransaction('Aborting...')
            if self.shouldProcess(dirName, foundFile, skip, validext):
                self.fileCount += 1
            else:
                continue
            if os.path.isdir(self.createPath(dirName, foundFile)):
                newFile = self.index(self.createPath(path, foundFile), drive, letter, validext, skip, kill=kill)
            else:
                newFile = self.createFileRef(drive, path, dirName, foundFile)
            if newFile:
                newFile[AEnums.PARENT] = fd[AEnums.ID]
                passedFiles.append(newFile)
        if not passedFiles and not root:
            return None
        for tmp in passedFiles:
            self.r.addFile(drive, tmp)
        return fd

    def getFile(self, filePath):
        return self.r.getFile(filePath)

    def remoteReady(self, d):
        if d.get(AEnums.REMOTEHOST, None) and d.get(AEnums.REMOTEPATH, None) and d[AEnums.TYPE] == self.CONTENT:
            return True
        else:
            return False

    def getFullPath(self, fileRef):
        return self.r.getFullPath(fileRef)

    def getSets(self):
        return self.r.getSets()

    @print_timing
    def hashify(self, drive):
        """
        Reduces drive to a dictionary of hashes for all its files, used for comparison of storage and content drives
        :param drive: the drive to be hashed
        :return: a dictionary with hash values / fileRef for all files in the drive
        """
        hashDict = {}
        for fileRef in self.r.getDriveFiles(drive):
            if fileRef[AEnums.DIR]:
                continue
            key = self.getKey(fileRef)
            if key in hashDict:
                hashDict[key], oldFile = self.chooseFileCopy(hashDict[key], fileRef)
            else:
                hashDict[key] = fileRef
        return hashDict

    def hashifyDuplicates(self, drive, hashDict=None):
        """
        Hashes a drive but also looks for duplicate files while doing so
        :param drive: hashes a drive but looks for duplicates
        :param hashDict: hash to compare for duplicates
        :return: the modified hash and a list of duplicate files
        """
        dupes = []
        hashDict = hashDict or {}
        for fileRef in self.r.getDriveFiles(drive):
            if fileRef[AEnums.DIR]:
                continue
            key = self.getKey(fileRef)
            if key in hashDict:
                hashDict[key], dupe = self.chooseFileCopy(hashDict[key], fileRef)
                dupes.append(dupe)
            else:
                hashDict[key] = fileRef
        return hashDict, dupes

    def chooseFileCopy(self, file1, file2):
        if self.r.isLocked(self.getBackupDrive(file1)):
            return file2, file1
        else:
            return file1, file2

    def getKey(self, fileRef):
        m = hashlib.md5()
        m.update(fileRef[AEnums.PATH].replace('{0}{1}'.format(os.sep, os.sep), os.sep).strip(os.sep).encode('UTF-16'))
        m.update(fileRef[AEnums.NAME].encode('UTF-16'))
        m.update(str(fileRef[AEnums.MTIME]))
        m.update(str(fileRef[AEnums.SIZE]))
        return m.hexdigest()

    @print_timing
    def reduceHash(self, drive, hashDict):
        """
        Takes a hash of files and removes any matching files found in the specified drive, used for
        finding files to restore / backup / remove
        :param drive: drive to reduce hash against
        :param hashDict: hash of existing files
        :return: reduced hash dictionary
        """
        hashd = self.hashify(drive)
        keys = set(hashDict.keys()) - set(hashd.keys())
        return {k: hashDict[k] for k in keys}

    @print_timing
    def hashDrives(self, toHash, toReduce):
        """
        Hashes all drives passed in toHash and then reduces it against all drives found in toReduce
        :param toHash: a list of drives to build a hash a against
        :param toReduce: a list of drives to reduce the hash against
        :return: a hash dictionary containing all files in toHash but no in toReduce
        """
        fileHash = {}
        for driveRef in toHash:
            fileHash.update(self.hashify(driveRef))
        for driveRef in toReduce:
            fileHash = self.reduceHash(driveRef, fileHash)
        files = fileHash.values()
        files = self.sortFiles(files)
        return files

    def getDuplicates(self):
        allDuplicates = []
        fileHash = {}
        for driveRef in self.getBackupDrives():
            if self.r.getRootFile(driveRef):
                fileHash, duplicates = self.hashifyDuplicates(driveRef, fileHash)
                if duplicates:
                    allDuplicates.extend(duplicates)
        return allDuplicates

    def getNewFiles(self):
        files = self.hashDrives(self.getContentDrives(), self.getBackupDrives())
        return files

    def getOldFiles(self):
        files = self.hashDrives(self.getBackupDrives(), self.getContentDrives())
        return files

    def getBackupDrive(self, fileRef):
        if self.backupMode == self.BALANCED:
            return self.getMostFreeDrive(self.BACKUP)
        else:
            for driveRef in self.getBackupDrives():
                if self.r.findDir(driveRef, fileRef[AEnums.PATH]):
                    return driveRef
        return None

    def getContentDrive(self, fileRef):
        for driveRef in self.getContentDrives():
            if self.r.findDir(driveRef, fileRef[AEnums.PATH]):
                return driveRef
        return None

    def getMostFreeDrive(self, driveType):
        if not driveType:
            driveType = backupManager.BACKUP
        mostFree = 0
        mostDrive = None
        drives = []
        if driveType == backupManager.BACKUP:
            drives = self.getBackupDrives()
        elif driveType == backupManager.CONTENT:
            drives = self.getContentDrives()
        for driveRef in drives:
            d = driveRef
            if d[AEnums.TYPE] == AEnums.BACKUP:
                if self.r.isLocked(d):
                    continue
            freeSpace = int(d[AEnums.FREESPACE]) - self.getPendingWrite(driveRef)
            if freeSpace > mostFree:
                mostDrive = driveRef
                mostFree = freeSpace
        return mostDrive

    def getPendingWrite(self, drive):
        return self._pendingWrite.get(drive[AEnums.NAME] + drive[AEnums.SERIAL], 0)

    def addPendingWrite(self, drive, fileDict):
        if self.isLocked(drive):
            return False
        key = drive[AEnums.NAME] + drive[AEnums.SERIAL]
        if (self._pendingWrite.get(key, 0) + fileDict[AEnums.SIZE]) > drive[AEnums.FREESPACE]:
            return False
        else:
            self._pendingWriteDirs.get(key, []).append(fileDict[AEnums.DIR])
            self._pendingWrite[key] = self._pendingWrite.get(key, 0) + fileDict[AEnums.SIZE]
            return True

    def addPendingDelete(self, drive, fileRef):
        key = drive[AEnums.NAME] + drive[AEnums.SERIAL]
        if key not in self._pendingWrite:
            self._pendingWrite[key] = 0
        self._pendingWrite[key] -= fileRef[AEnums.SIZE]

    def clearPendingWrite(self, drive=None):
        if drive:
            drives = [drive]
        else:
            drives = self.getBackupDrives() + self.getContentDrives()
        for drive in drives:
            key = drive[AEnums.NAME] + drive[AEnums.SERIAL]
            self._pendingWriteDirs[key] = []
            self._pendingWrite[key] = 0

    def isLocked(self, drive):
        return self.r.isLocked(drive)

    def batchFiles(self, files, kill):
        sortedFiles = {}
        for fileRef in files:
            if self.shouldExit(kill):
                break
            k = (fileRef[AEnums.VOLNAME], fileRef[AEnums.SERIAL])
            if k not in sortedFiles:
                sortedFiles[k] = {'files': []}
            sortedFiles[k]['files'].append(fileRef)
        return sortedFiles

    def getDriveLetter(self, key):
        letter = DriveUtil.getLetter(*key)
        while not letter:
                self.sendError(key[0] + "-" + key[1])
                answer = self.getErrorAnswer()
                if answer:
                    letter = DriveUtil.getLetter(*key)
                else:
                    letter = False
                    break
        return letter

    def copyFilesTo(self, files, path, kill=None):
        self.resetFileVars()
        errors = []
        self.total = len(files)
        sortedFiles = self.batchFiles(files, kill)
        self.calculateCopyTotal(sortedFiles)
        for key in sortedFiles:
            if self.shouldExit(kill):
                break
            self.currentDrive = '{0}-{1}'.format(*key)
            self.driveFinished += 1
            d = self.r.getDrive(*key + (self._setName,))
            letter = self.getDriveLetter(key)
            if not letter:
                errors.append('Cancelled copy from {0}'.format(key[0] + "-" + key[1]))
                continue
            for fileRef, fpath in zip(sortedFiles[key]['files'],
                                      self.r.getFullPaths(sortedFiles[key]['files'], driveType=d[AEnums.TYPE])):
                if self.shouldExit(kill):
                    break
                try:
                    self.count += 1
                    self.copyFrom = fpath
                    self.copyTo = path
                    self.currentFile = fileRef[AEnums.NAME]
                    self.copyFileAction(fpath, path, fileRef, d, errors)
                except AbortCopy:
                    pass
        if self.shouldExit(kill):
            errors.append('Copy aborted')
        return errors

    def shouldExit(self, event):
        if event and event.is_set():
            return True
        return False

    def batchRemoveFiles(self, toRemove, errorMessages, toStore, kill=None):
        for removeFile in toRemove:
            if self.shouldExit(kill):
                break
            buDrive = self.getDrive(removeFile)
            if not buDrive:
                continue
            key = buDrive[AEnums.NAME] + buDrive[AEnums.SERIAL]
            if self.r.isLocked(buDrive):
                error = "%s is locked, cannot remove %s!" % (buDrive[AEnums.NAME], removeFile[AEnums.NAME])
                errorMessages.append(error)
                logger.error(error)
                continue
            self.addPendingDelete(buDrive, removeFile)
            if key not in toStore:
                toStore[key] = {}
                toStore[key]['old'] = []
                toStore[key]['files'] = []
                toStore[key]['drive'] = buDrive
            toStore[key]['old'].append(removeFile)

    def batchCopyFiles(self, files, errorMessages, toStore, kill=None):
        for fileRef in files:
            if self.shouldExit(kill):
                break
            buDrive = self.getBackupDrive(fileRef) or self.getMostFreeDrive(backupManager.BACKUP)
            if buDrive:
                key = buDrive[AEnums.NAME] + buDrive[AEnums.SERIAL]
                if not self.addPendingWrite(buDrive, fileRef):
                    error = "Not enough space to store " + fileRef[
                        AEnums.NAME] + " on any backup drive, consider adding another backup drive?" + \
                        " Unlock backup drives with free space?"
                    errorMessages.append(error)
                    logger.error(error)
                    continue
            if not buDrive:
                error = "Error getting an open drive, no backup drives? All backup drives full? Locked?"
                errorMessages.append(error)
                logger.error(error)
                return errorMessages
            if self.r.isLocked(buDrive):
                error = "%s is locked, cannot backup %s!" % (buDrive[AEnums.NAME], fileRef[AEnums.NAME])
                errorMessages.append(error)
                logger.error(error)
                continue
            if key not in toStore:
                toStore[key] = {}
                toStore[key]['files'] = []
                toStore[key]['drive'] = buDrive
            toStore[key]['files'].append(fileRef)

    def batchRestoreFiles(self, toRestore, toStore, kill=None):
        for restoreFile in toRestore:
            if self.shouldExit(kill):
                break
            buDrive = self.getDrive(restoreFile)
            key = buDrive[AEnums.NAME] + buDrive[AEnums.SERIAL]
            if not buDrive:
                continue
            if key not in toStore:
                toStore[key] = {
                    'files': [],
                    'drive': buDrive,
                }
            toStore[key]['files'].append(restoreFile)

    def calculateCopyTotal(self, toStore):
        self.driveCount = len(toStore.keys())
        for fileRef in itertools.chain(*[x['files'] for x in toStore.values()]):
            self.copyTotal += fileRef[AEnums.SIZE]

    def resetFileVars(self):
        self.copyFrom = None
        self.copyTo = None
        self.currentFile = None
        self.currentDrive = None
        self.copyTotal = 0
        self.copyFinished = 0
        self.count = 0
        self.total = 0
        self.driveCount = 0
        self.driveFinished = 0
        self.fileCount = 0

    def getBackupLetter(self, buRef, missingDrives, driveId, errorMessages):
        backupLetter = None
        while not backupLetter:
            backupLetter = DriveUtil.getLetter(buRef[AEnums.NAME], buRef[AEnums.SERIAL])
            if not backupLetter:
                self.sendError(buRef[AEnums.NAME] + "-" + buRef[AEnums.SERIAL])
                answer = self.getErrorAnswer()
                if answer:
                    backupLetter = DriveUtil.getLetter(buRef[AEnums.NAME], buRef[AEnums.SERIAL])
                else:
                    missingDrives[buRef[AEnums.NAME] + buRef[AEnums.SERIAL]] = driveId
                    backupLetter = False
                    break
        if not backupLetter:
                error = buRef[AEnums.NAME] + "-" + buRef[
                    AEnums.SERIAL] + " not connected, skipping all files for this drive."
                errorMessages.append(error)
                logger.error(error)
                raise AbortDrive()
        return backupLetter

    def getSourceLetter(self, sourceName, sourceSerial, fileRef, errorMessages):
        sourceLetter = DriveUtil.getLetter(sourceName, sourceSerial)
        if not sourceLetter:
            error = sourceName + "-" + sourceSerial + " missing, skipping " + fileRef[AEnums.NAME]
            errorMessages.append(error)
            logger.error(error)
            raise AbortCopy()
        return sourceLetter

    def isMissing(self, buRef, missingDrives, fileRef, errorMessages):
        if (buRef[AEnums.NAME] + buRef[AEnums.SERIAL]) in missingDrives:
            error = buRef[AEnums.NAME] + "-" + buRef[AEnums.SERIAL] + " not connected, skipping " + fileRef[
                AEnums.NAME]
            errorMessages.append(error)
            logger.error(error)
            raise AbortDrive()
        return False

    def removeFile(self, toRemove, errorMessages):
        try:
            os.chmod(toRemove, stat.S_IWRITE)
            if os.path.exists(toRemove):
                os.unlink(toRemove)
        except Exception, e:
            error = "Error deleting " + toRemove + ": %s" % (str(e))
            errorMessages.append(error)
            logger.error(error)
            raise AbortCopy()

    def removeOldFiles(self, toStore, driveId, missingDrives, errorMessages, buRef, kill=None):
        for idx, fileRef in enumerate(toStore[driveId].get('old', [])):
            if self.shouldExit(kill):
                break
            self.currentFile = fileRef[AEnums.NAME]
            self.copyTo = None
            msg = "Removing old file " + fileRef[AEnums.NAME] + " on " + buRef[AEnums.NAME] + "... File " + str(
                self.count) + "/" + str(self.total)
            self.sendMessage(msg)
            logger.info(msg)
            try:
                self.isMissing(buRef, missingDrives, fileRef, errorMessages)
                backupLetter = self.getBackupLetter(buRef, missingDrives, driveId, errorMessages)

                # noinspection PyTypeChecker
                toRemove = self.fullPath(backupLetter + ":\\", buRef[AEnums.SETNAME], fileRef[AEnums.PATH],
                                         fileRef[AEnums.NAME])
                self.copyFrom = toRemove
                self.removeFile(toRemove, errorMessages)
            except AbortCopy:
                self.count += 1
                continue
            except AbortDrive:
                self.count += len(toStore[driveId].get('old', [])) - idx
                break
            self.count += 1
            self.r.updateFreeSpace(buRef)

    def createDirs(self, buRef, targetDir, fileRef, errorMessages):
        if not os.path.exists(self.toFullPath(targetDir)):
            # noinspection PyBroadException
            try:
                os.makedirs(targetDir)
            except:
                error = buRef[AEnums.NAME] + "-" + buRef[AEnums.SERIAL] + " can't create dir, skipping " + \
                    fileRef[AEnums.NAME]
                errorMessages.append(error)
                logger.error(error)
                raise AbortCopy()
        return True

    def copyFileAction(self, source, targetDir, fileRef, buRef, errorMessages):
        try:
            shutil.copy2(source, self.fullPath(targetDir, fileRef[AEnums.NAME]))
        except IOError, e:
            error = buRef[AEnums.NAME] + "-" + buRef[AEnums.SERIAL] + " can't copy source, skipping " + fileRef[
                AEnums.NAME] + ": %s" % e
            errorMessages.append(error)
            logger.error(error)
            raise AbortCopy()
        finally:
            self.copyFinished += fileRef[AEnums.SIZE]
        return True

    def getCopyPaths(self, sourceLetter, backupLetter, fileRef, buRef, errorMessages, restore=False):
        try:
            if not restore:
                self.copyFrom = source = self.fullPath(sourceLetter + ":", fileRef[AEnums.PATH],
                                                       fileRef[AEnums.NAME])
                # noinspection PyTypeChecker
                self.copyTo = targetDir = self.createPath(backupLetter + ":", buRef[AEnums.SETNAME],
                                                          fileRef[AEnums.PATH])
            else:
                self.copyTo = targetDir = self.createPath(sourceLetter + ":", fileRef[AEnums.PATH])
                # noinspection PyTypeChecker
                self.copyFrom = source = self.fullPath(backupLetter + ":", buRef[AEnums.SETNAME],
                                                       fileRef[AEnums.PATH], fileRef[AEnums.NAME])
        except UnicodeDecodeError, e:
            error = buRef[AEnums.NAME] + "-" + buRef[
                AEnums.SERIAL] + " can't copy source due to naming issue, skipping: %s" % e
            errorMessages.append(error)
            raise AbortCopy()
        return source, targetDir

    def copyNewFiles(self, toStore, driveId, missingDrives, errorMessages, buRef, kill=None):
        msg = "Backing up new files on " + buRef[AEnums.NAME] + "..."
        self.sendMessage(msg)
        logger.info(msg)
        fileList = toStore[driveId].get('files', [])
        for idx, fileRef in enumerate(fileList):
            if self.shouldExit(kill):
                break
            try:
                self.count += 1
                self.currentFile = fileRef[AEnums.NAME]
                self.copyFrom = self.copyTo = None
                msg = "Backing up " + fileRef[AEnums.NAME] + " to " + buRef[AEnums.NAME] + "... File " + str(
                    self.count) + "/" + str(self.total)
                self.sendMessage(msg)
                logger.info(msg)
                sourceSerial = fileRef[AEnums.SERIAL]
                sourceName = fileRef[AEnums.VOLNAME]
                sourceLetter = self.getSourceLetter(sourceName, sourceSerial, fileRef, errorMessages)
                self.isMissing(buRef, missingDrives, fileRef, errorMessages)
                backupLetter = self.getBackupLetter(buRef, missingDrives, driveId, errorMessages)
                source, targetDir = self.getCopyPaths(sourceLetter, backupLetter, fileRef, buRef, errorMessages)
                self.createDirs(buRef, targetDir, fileRef, errorMessages)
                self.copyFileAction(source, targetDir, fileRef, buRef, errorMessages)
            except AbortCopy:
                continue
            except AbortDrive:
                self.copyFinished += sum([x[AEnums.SIZE] for x in fileList][idx:])
                self.count += len(fileList) - idx+1
                break
            self.r.updateFreeSpace(buRef)

    def prioritizeDrive(self, toStore):
        order = []
        for driveId in toStore.keys():
            buRef = toStore[driveId]['drive']
            if DriveUtil.getLetter(buRef[AEnums.NAME], buRef[AEnums.SERIAL]):
                order.append(driveId)
        [order.append(x) for x in toStore.keys() if x not in order]
        return order

    def backupNewFiles(self, kill=None):
        self.resetFileVars()
        files = self.getNewFiles()
        toRemove = self.getOldFiles() + self.getDuplicates()
        toStore = {}
        msg = "Sorting files..."
        self.sendMessage(msg)
        logger.info(msg)
        missingDrives = {}
        errorMessages = []
        self.total = len(files) + len(toRemove)
        self.clearPendingWrite()
        self.batchRemoveFiles(toRemove, errorMessages, toStore, kill=kill)
        self.batchCopyFiles(files, errorMessages, toStore, kill=kill)
        self.calculateCopyTotal(toStore)
        driveOrder = self.prioritizeDrive(toStore)
        for driveId in driveOrder:
            if self.shouldExit(kill):
                break
            buRef = toStore[driveId]['drive']
            self.currentDrive = '{0}-{1}'.format(buRef[AEnums.NAME], buRef[AEnums.SERIAL])
            self.driveFinished += 1
            #backupLetter = DriveUtil.getLetter(buRef[AEnums.NAME], buRef[AEnums.SERIAL])
            self.removeOldFiles(toStore, driveId, missingDrives, errorMessages, buRef, kill=kill)
            self.copyNewFiles(toStore, driveId, missingDrives, errorMessages, buRef, kill=kill)
            msg = "Refreshing " + buRef[AEnums.NAME] + "-" + buRef[AEnums.SERIAL] + "..."
            self.sendMessage(msg)
            logger.info(msg)
            self.refreshDrive(buRef)
        if self.shouldExit(kill):
            errorMessages.append('Backup Aborted')
        return errorMessages

    def getTargetContentDrive(self, fileRef, errorMessages):
        mostFree = False
        contentRef = self.getContentDrive(fileRef)
        if not contentRef:
            mostFree = True
            contentRef = self.getMostFreeDrive(backupManager.CONTENT)
        if contentRef and not self.addPendingWrite(contentRef, fileRef):
            if not mostFree:
                contentRef = self.getMostFreeDrive(backupManager.CONTENT)
            if not self.addPendingWrite(contentRef, fileRef):
                contentRef = None
        if not contentRef:
            error = "Cannot find a content drive with enough space to store " + fileRef[
                    AEnums.NAME] + ", all drives full?"
            errorMessages.append(error)
            logger.error(error)
            raise AbortCopy()
        return contentRef

    def restoreFiles(self, toStore, driveId, buRef, missingDrives, drivesToRefresh, errorMessages, kill=None):
        for idx, fileRef in enumerate(toStore[driveId]['files']):
            self.count += 1
            self.currentFile = fileRef[AEnums.NAME]
            self.copyTo = self.copyFrom = None
            if self.shouldExit(kill):
                break
            try:
                contentRef = self.getTargetContentDrive(fileRef, errorMessages)
                contentLetter = self.getSourceLetter(contentRef[AEnums.NAME], contentRef[AEnums.SERIAL],
                                                     fileRef, errorMessages)
                msg = "Restoring " + fileRef[AEnums.NAME] + " to " + contentRef[AEnums.NAME] + "... File " + str(
                      self.count) + "/" + str(self.total)
                self.sendMessage(msg)
                logger.info(msg)
                self.isMissing(buRef, missingDrives, fileRef, errorMessages)
                backupLetter = self.getBackupLetter(buRef, missingDrives, driveId, errorMessages)
                # noinspection PyTypeChecker
                source, targetDir = self.getCopyPaths(contentLetter, backupLetter, fileRef, buRef, errorMessages,
                                                      restore=True)
                self.createDirs(contentRef, targetDir, fileRef, errorMessages)
                self.copyFileAction(source, targetDir, fileRef, contentRef, errorMessages)
                if contentRef not in drivesToRefresh:
                    drivesToRefresh.append(contentRef)
            except AbortCopy:
                continue
            except AbortDrive:
                self.copyFinished += sum([x[AEnums.SIZE] for x in toStore[driveId]['files']][idx:])
                self.count += len(toStore[driveId]['files']) - idx+1
                break
            self.r.updateFreeSpace(contentRef)

    def doRefresh(self, drives, errorMessages, kill=None):
        for d in drives:
            msg = "Refreshing " + d[AEnums.NAME] + "-" + d[AEnums.SERIAL] + "..."
            self.sendMessage(msg)
            logger.info(msg)
            if self.remoteReady(d):
                self.remoteRefresh(d, errorMessages, kill=kill)
            else:
                self.refreshDrive(d, kill=kill)

    # noinspection PyBroadException
    def restoreOldFiles(self, kill=None):
        """
        Gets a list of files on the backup drives but not on the content drives and restores them
        :return: None
        """
        self.resetFileVars()
        toRestore = self.getOldFiles()
        toStore = {}
        self.total = len(toRestore)
        msg = "Sorting files..."
        self.sendMessage(msg)
        logger.info(msg)
        missingDrives = {}
        drivesToRefresh = []
        errorMessages = []

        self.batchRestoreFiles(toRestore, toStore, kill=kill)
        self.calculateCopyTotal(toStore)
        for driveId in toStore.keys():
            if self.shouldExit(kill):
                break
            buRef = toStore[driveId]['drive']
            self.currentDrive = '{0}-{1}'.format(buRef[AEnums.NAME], buRef[AEnums.SERIAL])
            self.driveFinished += 1
            msg = "Restoring files from " + buRef[AEnums.NAME] + "..."
            self.sendMessage(msg)
            logger.info(msg)
            self.restoreFiles(toStore, driveId, buRef, missingDrives, drivesToRefresh, errorMessages, kill=kill)

        self.doRefresh(drivesToRefresh, errorMessages, kill=kill)
        if self.shouldExit(kill):
            errorMessages.append('Restore Aborted')
        return errorMessages

    def remoteRefresh(self, d, errorMessages, kill=None):
        msg = 'Remote indexing {0}-{1}'.format(d[AEnums.NAME], d[AEnums.SERIAL])
        logger.info(msg)
        self.sendMessage(msg)
        result = self.remoteIndexDrive(d, kill=kill)
        logger.info('Remote call returned.')
        if not result:
            error = 'Error remote indexing {0}-{1}, unknown error'.format(d[AEnums.NAME], d[AEnums.SERIAL])
            logger.error(error)
            self.sendMessage(error)
            errorMessages.append(error)
        elif result == AEnums.NOTEXIST:
            error = 'Remote path for {0}-{1} does not exist: {2}'.format(d[AEnums.NAME], d[AEnums.SERIAL],
                                                                         d[AEnums.REMOTEPATH])
            logger.error(error)
            self.sendMessage(error)
            errorMessages.append(error)
        elif result == AEnums.ABORT:
            pass
        else:
            msg = 'Remote indexing done for {0}-{1}'.format(d[AEnums.NAME], d[AEnums.SERIAL])
            logger.info(msg)
            self.sendMessage(msg)
            self.r.updateFreeSpace(d)

    def localRefresh(self, d, errorMessages, kill=None):
        result = self.refreshDrive(d, kill=kill)
        while result == AEnums.MISSING:
            self.sendError(d[AEnums.NAME] + "-" + d[AEnums.SERIAL])
            answer = self.getErrorAnswer()
            if answer:
                result = self.refreshDrive(d, kill=kill)
            else:
                error = 'Could not refresh {0}-{1}, drive is not connected'.format(d[AEnums.NAME],
                                                                                   d[AEnums.SERIAL])
                logger.error(error)
                self.sendMessage(error)
                errorMessages.append(error)
                raise AbortDrive()
        if not self.shouldExit(kill):
            if not result:
                error = 'Could not refresh {0}-{1}, unknown error'.format(d[AEnums.NAME], d[AEnums.SERIAL])
                logger.error(error)
                self.sendMessage(error)
                errorMessages.append(error)
            else:
                msg = "Refreshed " + d[AEnums.NAME] + "-" + d[AEnums.SERIAL] + "!"
                logger.info(msg)
                self.sendMessage(msg)
                self.r.updateFreeSpace(d)

    def refresh(self, drives, remote=False, kill=None):
        errorMessages = []
        for d in drives:
            if self.shouldExit(kill):
                break
            try:
                if remote and self.remoteReady(d):
                    self.remoteRefresh(d, errorMessages, kill=kill)
                else:
                    msg = 'Refreshing {0}-{1}'.format(d[AEnums.NAME], d[AEnums.SERIAL])
                    logger.info(msg)
                    self.sendMessage(msg)
                    self.localRefresh(d, errorMessages, kill=kill)
            except AbortDrive:
                pass
        if self.shouldExit(kill):
            errorMessages.append('Refresh Aborted')
        return errorMessages

    def export(self, driveType, path):
        drives = []
        if AEnums.BACKUP in driveType:
            drives += self.getBackupDrives()
        if AEnums.CONTENT in driveType:
            drives += self.getContentDrives()
        self.r.export(drives, path, self._setName)

    def getChkdskLetter(self, drive, errormessages):
        letter = None
        while not letter:
            letter = DriveUtil.getLetter(drive[AEnums.NAME], drive[AEnums.SERIAL])
            if not letter:
                self.sendError(drive[AEnums.NAME] + "-" + drive[AEnums.SERIAL])
                answer = self.getErrorAnswer()
                if answer:
                    letter = DriveUtil.getLetter(drive[AEnums.NAME], drive[AEnums.SERIAL])
                else:
                    letter = False
                    break
        if not letter:
            errormessages.append('{0} - {1} not hooked up, could not run chkdsk.'.format(
                drive[AEnums.NAME], drive[AEnums.SERIAL]
            ))
        return letter

    def runChkdsk(self, letter, kill):
        chkdskOut = ''
        p = subprocess.Popen(['cmd.exe', '/C', 'chkdsk /F/X/I/C %s:' % letter],
                             stdout=subprocess.PIPE, bufsize=0)
        while p.poll() is None:
            if self.shouldExit(kill):
                subprocess.call('TASKKILL /F /IM chkdsk.exe')
                return None, None
            sleep(1)
            chkdskOut += p.stdout.readline()
        chkdskOut += ''.join(p.stdout.readlines())
        p.wait()
        m = re.search('(\d*) KB in bad sectors', chkdskOut)
        badSectors = False
        if m and m.group(1) != '0':
            badSectors = True
        result = p.returncode
        return badSectors, result

    def chkdsk(self, driveRefs, kill=None):
        self.driveCount = len(driveRefs)
        self.driveFinished = 0
        errormessages = []
        for drive in driveRefs:
            self.driveFinished += 1
            if self.shouldExit(kill):
                break
            letter = self.getChkdskLetter(drive, errormessages)
            if not letter:
                continue
            # noinspection PyBroadException
            try:
                badSectors, result = self.runChkdsk(letter, kill)
                if (badSectors, result) == (None, None):
                    continue
                if result in [0, 1]:
                    if result == 1:
                        errormessages.append('{0} - {1} had errors that were fixed.'.format(
                            drive[AEnums.NAME], drive[AEnums.SERIAL]
                        ))
                    if badSectors:
                        errormessages.append(
                            '{0} - {1} has bad sectors, please run chkdsk manually to attempt to recover them.'.format(
                            drive[AEnums.NAME], drive[AEnums.SERIAL]))
                    drive[AEnums.CHKDSK] = datetime.date.today().isoformat()
                    self.r.updateFreeSpace(drive)
                elif result == 3:
                    if DriveUtil.getDriveType(letter) == 4:
                        drive[AEnums.CHKDSK] = datetime.date.today().isoformat()
                        self.r.updateFreeSpace(drive)
                        errormessages.append('{0} - {1} is a network drive.'.format(
                            drive[AEnums.NAME], drive[AEnums.SERIAL]
                        ))
                    else:
                        errormessages.append('{0} - {1} unknown error.'.format(
                            drive[AEnums.NAME], drive[AEnums.SERIAL]
                        ))
                else:
                    errormessages.append('Unknown result for {0} - {1}: {2}'.format(
                        drive[AEnums.NAME], drive[AEnums.SERIAL], result
                    ))
            except:
                print traceback.format_exc()
                errormessages.append(traceback.format_exc())
        if self.shouldExit(kill):
            errormessages.append('Chkdsk aborted.')
        return errormessages

    def getToday(self):
        return datetime.date.today()

    def toChkdsk(self):
        if not self.nagLength:
            return []
        ret = []
        for drive in self.getBackupDrives() + self.getContentDrives():
            try:
                date = datetime.datetime.strptime(drive[AEnums.CHKDSK], '%Y-%m-%d').date()
            except ValueError:
                date = datetime.date.min
            if (self.getToday() - date).days > self.nagLength*30:
                ret.append(drive)
        return ret
